import threading
import traceback
import urllib.error
import urllib.parse
import urllib.request
from random import shuffle

class ProxyGenerator(object):
  def __init__(self):
    self.pages = []  # Page urls
    self._lock = threading.Lock()
    self._itr = None

  def NextProxy(self):
    self._lock.acquire()
    try:
      value = self._itr.__next__()
    except(StopIteration, AttributeError):
      value = None  # Returning None means not using proxy.
      self._itr = self.GetProxyIterator()
    finally:
      self._lock.release()
      return value

  # Returns a generator for all proxies found in all pages.
  def GetProxyIterator(self):
    shuffle(self.pages)
    for url in self.pages:
      webdata = self._TryGetWebData(url)
      if webdata:
        for proxy in self.GetIps(webdata):
          yield proxy

  # Returns a generator for proxies found in @webdata.
  def GetIps(self, webdata):
    pass

  def _TryGetWebData(self, url):
    print('Getting proxies in page ' + url)
    try:
      webdata = urllib.request.urlopen(url).read()
      return webdata
    except:
      print("Error occurs while reading " + url)
      print(traceback.print_exc())
      return None
      # sys.exit(1)
